# Scrapy settings for dragonforum project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/topics/settings.html
#
# Or you can copy and paste them from where they're defined in Scrapy:
# 
#     scrapy/conf/default_settings.py
#

BOT_NAME = 'dragonspider'
BOT_VERSION = '1.0'

SPIDER_MODULES = ['dragonspider.spiders']
NEWSPIDER_MODULE = 'dragonspider.spiders'
DEFAULT_ITEM_CLASS = 'dragonspider.items.Topic'
ITEM_PIPELINES = ['dragonspider.pipelines.DragonPipeline']
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
DOWNLOAD_DELAY = 0.1
#LOG_LEVEL = 'INFO'
DOWNLOADER_MIDDLEWARES = {
    # Engine side
    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': None, # breaks crawling thread pages
    'dragonspider.middlewares.NoDupsDownloaderMiddleware': 1000,
    # Downloader side
}

DJANGO_SERVER = 'http://localhost:8000'
#DJANGO_SERVER = 'http://simple-dm-forums.dbenamy.webfactional.com'
